import datetime
import json
import re

import requests
from pyquery import PyQuery as pyquery
from dateutil.relativedelta import relativedelta
from lxml import etree
from applications.extensions.init_redis import redis_client
from applications.extensions import db
from applications.models import FundStockListCwzx
from flask import current_app


class Crawl_cwzx(object):
    def __init__(self):
        self.req_url = 'http://cwzx.shdjt.com/top500.asp?pd=1'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46'
        }
        res = requests.get('https://xueqiu.com', headers=self.headers)
        cookies_dict = res.cookies.get_dict()
        cookies_str = ';'.join([key + '=' + value for key, value in cookies_dict.items()])
        self.headers['Cookie'] = cookies_str

    def run(self):
        top_page = requests.get(self.req_url).content.decode('gbk')
        bs_content = etree.HTML(top_page)
        links = bs_content.xpath('//table[@class="tb2td1"]//table//tr[@height="30"]/td/a')

        today = datetime.date.today()
        dt = datetime.datetime.now()
        i = 0
        for link in links:
            if link.xpath('./@href') is None or len(link.xpath('./@href')) == 0:
                current_app.logger.error(link.xpath('./@href'))
                print(link.xpath('./@href'))
                continue
            href = link.xpath('./@href')[0]
            title = link.xpath('./@title')[0]
            url = "http://cwzx.shdjt.com/%s" % href
            current_app.logger.info('title=%s,url=%s' % (title, url))
            self.parse_fund(dt, today, url)

            i = i + 1
            if i > 390:
                break

    def parse_fund(self, dt, today, url):
        fund_content = requests.get(url).content.decode('gbk')
        fund_content = etree.HTML(fund_content)
        stock_trs = fund_content.xpath('//table[@class="tb0td1"]//tr')
        j = 0
        for stock_tr in stock_trs:
            if j > 0:
                tds = stock_tr.xpath('./td')
                # 序号
                id = tds[0].xpath('./text()')[0]
                # 股票代码
                code = tds[1].xpath('./text()')[0]
                code = code.strip()
                # 股票名称
                name = tds[2].xpath('./text()')[0]
                name = name.strip()

                # 原始价格
                if len(tds[3].xpath('./text()')) == 0:
                    continue
                original_price = tds[3].xpath('./text()')[0]

                # 当前价格
                if len(tds[4].xpath('./text()')) == 0:
                    continue
                cur_price = tds[4].xpath('./text()')[0]

                # 涨跌幅
                if len(tds[5].xpath('./text()')) == 0:
                    continue
                rises = tds[5].xpath('./text()')[0]

                # 更新日期
                if len(tds[7].xpath('./text()')) == 0:
                    continue
                date = tds[7].xpath('./text()')[0]

                # 持仓占比
                if len(tds[12].xpath('./text()')) == 0:
                    continue
                cgbl = tds[12].xpath('./text()')[0]

                # 增减持
                if len(tds[14].xpath('./text()')) == 0:
                    continue
                zjc = tds[14].xpath('./text()')[0]

                time1 = datetime.datetime.strptime(date, '%Y/%m/%d').date()
                new_date = time1 + relativedelta(months=6)
                rises_n = rises[:-1] if rises is not None else None
                if ',' in rises_n:
                    rises_n = rises_n.replace(',', '')

                if rises_n is not None and float(rises_n) > -30 and 60 > float(cur_price) \
                        and new_date > today and zjc != '减少' and zjc != '未变':
                    old_code = code
                    subCode = code[0:2]
                    if subCode == '60' or subCode == '68' or subCode == '90':
                        code = 'SH' + code
                    else:
                        code = 'SZ' + code
                    key = code + name

                    xq_url = 'https://xueqiu.com/S/%s' % code

                    value = redis_client.get(key)
                    if value is None:
                        redis_client.set(key, name, ex=1800)
                        current_app.logger.info('id=%s,code=%s,name=%s,original_price=%s,cur_price=%s,rises=%s,date=%s,cgbl=%s,zjc=%s' %
                              (id, code, name, original_price, cur_price, rises, date, cgbl, zjc))

                        # 股票财务数据,基础每股收益,主营业务收入增长率(%) 净利润增长率(%) 净资产增长率(%) 总资产增长率(%)
                        basic_eps1, m_busi_income1, m_busi_income2, n_assert_grate1, n_assert_grate2, n_profit_grate1, \
                            n_profit_grate2, t_assert_grate1, t_assert_grate2 = self.crawl_stock_indicator(code)

                        # 股票基础信息
                        pe_ttm = self.crawl_stock_quote(code)

                        # 千股千评,今日总评 机构参与度 控盘程度 平均仓位 资金动向 行业 价值评估
                        t_grade, participation, manipulate, control_level, average_posi, industry, value_assessment = \
                            self.crawl_stock_qgqp(old_code)

                        if t_grade == 0 or participation == 0 or manipulate == '' or control_level == '' \
                                or average_posi == '' or industry == '' or value_assessment == '':
                            continue

                        # 百度股市通，平均价格，最高价格，最低价格，机构持仓数
                        avg_price, max_price, min_price, organ_num = self.crawl_stock_gst(old_code)
                        fund_stock_cwzx = FundStockListCwzx(create_time=dt, update_time=dt, create_date=today,
                                                            symbol=old_code,
                                                            name=name, stock_name=name, stock_code=old_code,
                                                            xq_symbol=old_code, xq_url=xq_url, percent=cgbl,
                                                            cur_price=cur_price,
                                                            original_price=original_price, rises=rises_n,
                                                            m_busi_income1=m_busi_income1,
                                                            n_profit_grate1=n_profit_grate1,
                                                            n_assert_grate1=n_assert_grate1,
                                                            t_assert_grate1=t_assert_grate1,
                                                            m_busi_income2=m_busi_income2,
                                                            n_profit_grate2=n_profit_grate2,
                                                            n_assert_grate2=n_assert_grate2,
                                                            t_assert_grate2=t_assert_grate2,
                                                            t_grade=t_grade, participation=participation,
                                                            manipulate=manipulate, control_level=control_level,
                                                            average_posi=average_posi, industry=industry,
                                                            value_assessment=value_assessment, basic_eps=basic_eps1,
                                                            organ_num=organ_num, avg_price=avg_price,
                                                            min_price=min_price,
                                                            max_price=max_price, pe_ttm=pe_ttm)
                        self.save(fund_stock_cwzx)

                        # stockSql = "INSERT INTO djfund.fund_stock_list_cwzx (create_time, update_time, create_date, " \
                        #            "symbol, name, stock_name, stock_code, xq_symbol, xq_url, percent, " \
                        #            "cur_price, original_price, rises, m_busi_income1, n_profit_grate1, n_assert_grate1, t_assert_grate1," \
                        #            "m_busi_income2, n_profit_grate2, n_assert_grate2, t_assert_grate2," \
                        #            "t_grade, participation, manipulate, control_level, average_posi, industry, value_assessment, basic_eps," \
                        #            "organ_num, avg_price, min_price, max_price) " \
                        #            "VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %s, %s, %s," \
                        #            "'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %s, %s, '%s', '%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s')" \
                        #            % (dt, dt, today, old_code, name, name, old_code, old_code, '', cgbl, cur_price,
                        #               original_price, rises_n, m_busi_income1, n_profit_grate1,
                        #               n_assert_grate1, t_assert_grate1, m_busi_income2, n_profit_grate2,
                        #               n_assert_grate2, t_assert_grate2, t_grade, participation, manipulate, control_level,
                        #               average_posi,
                        #               industry, value_assessment, basic_eps1, organ_num, avg_price, min_price, max_price)
            j = j + 1

    def crawl_stock_quote(self, code):
        quote = requests.get(
            'https://stock.xueqiu.com/v5/stock/quote.json?symbol=%s&extend=detail' % code,
            headers=self.headers)
        quoteData = json.loads(quote.text)
        pe_ttm = quoteData['data']['quote']['pe_ttm']
        return pe_ttm

    def crawl_stock_gst(self, old_code):
        scode = old_code
        # 机构预测
        url = 'https://gushitong.baidu.com/opendata?openapi=1&dspName=iphone&tn=tangram&client=app&query=%s' \
              '&code=%s&word=%s&resource_id=5429&ma_ver=4&finClientType=pc' % (scode, scode, scode)
        response = requests.get(url)
        try:
            res_json = json.loads(response.text)
        except Exception as e:
            res_json = None
            current_app.logger.error(url, response.content)
            print(e)

        resultData = res_json['Result'][3]['DisplayData']['resultData'] if res_json is not None else []
        organ_num, avg_price, min_price, max_price = 0, 0, 0, 0
        if len(resultData) > 0 and 'organRating' in resultData['tplData']['result']['tabs'][5]['content']['newCompany']:
            organRating = resultData['tplData']['result']['tabs'][5]['content']['newCompany'][
                'organRating']
            organ_num = organRating['organNum']
            if organ_num == '0':
                pass
            else:
                avg_price = organRating['avgPrice']
                min_price = organRating['minPrice']
                max_price = organRating['maxPrice']
        return avg_price, max_price, min_price, organ_num

    def crawl_stock_qgqp(self, old_code):
        response1 = requests.get('http://qgqp.shdjt.com/gpdm.asp?gpdm={0}'.format(old_code))
        doc1 = pyquery(response1.text)
        tables = doc1('table.tb2td1').items()
        # 今日总评 机构参与度 控盘程度 平均仓位 资金动向 行业 价值评估
        t_grade, participation, manipulate, control_level, average_posi, industry, value_assessment = 0, 0, '', '', '', '', ''
        n = 0
        for table in tables:
            if n == 5:
                dianping = table('tr td.tdleft p').text()
                if re.findall(r'.今日总评(.*)分.', dianping).__len__() == 0:
                    continue
                t_grade = re.findall(r'.今日总评(.*)分.', dianping)[0]

                if re.findall(r'.机构参与度为(.*)%,属于(.*),.', dianping).__len__() == 0:
                    continue
                participation = re.findall(r'.机构参与度为(.*)%,属于(.*),.', dianping)[0][0]
                manipulate = re.findall(r'.机构参与度为(.*)%,属于(.*),.', dianping)[0][1]

                if re.findall(r'.平均仓位为(.*)，市场.', dianping).__len__() == 0:
                    continue
                control_level = re.findall(r'.平均仓位为(.*)，市场.', dianping)[0]

                if re.findall(r'.行业资金呈现为(.*)。.', dianping).__len__() == 0:
                    continue
                average_posi = re.findall(r'.行业资金呈现为(.*)。.', dianping)[0]

                if re.findall(r'.价值评估：属于(.*),.', dianping).__len__() == 0:
                    continue
                industry = re.findall(r'.价值评估：属于(.*),.', dianping)[0]

                if re.findall(r'.价值评估：(.*)', dianping).__len__() == 0:
                    continue
                value_assessment = re.findall(r'.价值评估：(.*)', dianping)[0]

            n += 1
        return t_grade, participation, manipulate, control_level, average_posi, industry, value_assessment

    def crawl_stock_indicator(self, code):
        # 主营业务收入增长率(%) 净利润增长率(%) 净资产增长率(%) 总资产增长率(%)
        m_busi_income1, n_profit_grate1, n_assert_grate1, t_assert_grate1 = None, None, 0, 0
        m_busi_income2, n_profit_grate2, n_assert_grate2, t_assert_grate2 = None, None, 0, 0
        # 基础每股收益
        basic_eps1 = None
        indicator = requests.get(
            'https://stock.xueqiu.com/v5/stock/finance/cn/indicator.json?symbol=%s&type=all&is_detail=true'
            '&count=5&timestamp=' % code, headers=self.headers)
        indicatorData = json.loads(indicator.text)
        indicatorList = indicatorData['data']['list']
        if indicatorList is not None and len(indicatorList) > 1:
            indicator1 = indicatorList[0]
            indicator2 = indicatorList[1]
            # indicator3 = indicatorList[2]

            np_atsopc_nrgal_yoy1 = indicator1['np_atsopc_nrgal_yoy']
            operating_income_yoy1 = indicator1['operating_income_yoy']
            basic_eps1 = indicator1['basic_eps']
            avg_roe1 = indicator1['avg_roe']
            # 扣非净利润同比增长
            nrgal_yoy1 = 0 if np_atsopc_nrgal_yoy1[0] is None else np_atsopc_nrgal_yoy1[0]
            # 营业收入同比增长
            income_yoy1 = 0 if operating_income_yoy1[0] is None else operating_income_yoy1[0]
            # 每股收益
            basic_eps1 = 0 if basic_eps1[0] is None else basic_eps1[0]
            # 净资产收益率
            avg_roe1 = 0 if avg_roe1[0] is None else avg_roe1[0]
            n_profit_grate1 = round(nrgal_yoy1, 2)
            m_busi_income1 = round(income_yoy1, 2)
            basic_eps1 = round(basic_eps1, 2)
            avg_roe1 = round(avg_roe1, 2)
            n_assert_grate1 = avg_roe1

            np_atsopc_nrgal_yoy2 = indicator2['np_atsopc_nrgal_yoy']
            operating_income_yoy2 = indicator2['operating_income_yoy']
            basic_eps2 = indicator2['basic_eps']
            avg_roe2 = indicator2['avg_roe']
            # 扣非净利润同比增长
            nrgal_yoy2 = 0 if np_atsopc_nrgal_yoy2[0] is None else np_atsopc_nrgal_yoy2[0]
            # 营业收入同比增长
            income_yoy2 = 0 if operating_income_yoy2[0] is None else operating_income_yoy2[0]
            # 每股收益
            basic_eps2 = 0 if basic_eps2[0] is None else basic_eps2[0]
            # 净资产收益率
            avg_roe2 = 0 if avg_roe2[0] is None else avg_roe2[0]
            n_profit_grate2 = round(nrgal_yoy2, 2)
            m_busi_income2 = round(income_yoy2, 2)
            avg_roe2 = round(avg_roe2, 2)
            n_assert_grate2 = avg_roe2
        return basic_eps1, m_busi_income1, m_busi_income2, n_assert_grate1, n_assert_grate2, n_profit_grate1, n_profit_grate2, t_assert_grate1, t_assert_grate2

    def save(self, fund_stock_cwzx: FundStockListCwzx):
        db.session.add(fund_stock_cwzx)
        db.session.commit()


if __name__ == '__main__':
    Crawl_cwzx().run()
